package com.lean.reptile.spider.sp;

import com.lean.reptile.spider.common.MyDownloader;
import com.lean.reptile.spider.common.MySpider;
import com.lean.reptile.spider.common.RedisScheduler;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Request;

@Component
public class ZonghengTask {

    private static final String SITE_CODE = "zongheng";

    private static final String URL = "http://book.zongheng.com/store/c0/c0/b0/u0/p";

    public void doTask() {
        MySpider mySpider = MySpider.create(new ZonghengProcessor());
        //自定义下载器，将下载失败的url记录到redis中
        mySpider.setDownloader(new MyDownloader(SITE_CODE));
        mySpider.setScheduler(new RedisScheduler(SITE_CODE));
        //持久化
        mySpider.addPipeline(new ZonghengPipeline());
        mySpider.thread(10);
        int totalPage = 999;
        // 添加起始url
        for(int i=1; i<=totalPage; i++) {
            Request request = new Request(URL + i+"/v9/s9/t0/u0/i1/ALL.html");
            // 在Request额外信息中设置页面类型
            request.putExtra(ZonghengProcessor.TYPE, ZonghengProcessor.LIST_TYPE);
            mySpider.addRequest(request);
        }
        //设置完成时退出
        mySpider.setExitWhenComplete(true);
        mySpider.run();
        mySpider.stop();
    }
}
